{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#卷积层\n",
    "#用3个3x3的过滤器去扫描6x6的图像,每个过滤器都会得到4x4的扫描结果,每个像素等于如下\n",
    "#z = w1*x1 + w2*x2 + ... + w9*x9 + b\n",
    "#a = a(z)\n",
    "\n",
    "#池化层\n",
    "#用一个2x2的最大池去扫描4x4的扫描结果,会得到一个2x2的池化结果,每个像素等于如下\n",
    "#池化层不存在学习参数\n",
    "#z = max(x1,x2,x3,x4)\n",
    "#a = z\n",
    "\n",
    "#输出层\n",
    "#用3个神经元做one hot输出.这里是全连接网络\n",
    "#z = w1*x1 + w2*x2 + w3*x3 + w4*x4 + b\n",
    "#a = a(z)\n",
    "\n",
    "#误差函数\n",
    "#C = 1/2 * [(y1 - a1)^2 + (y2 - a2)^2 + (y3 - a3)^2] <- 对所有x求和\n",
    "\n",
    "#定义符号delta = dC/dz\n",
    "\n",
    "#sigmoid函数求导\n",
    "#da/dz = a(z) * (1 - a(z))\n",
    "\n",
    "#计算输出层的delta\n",
    "#delta = dC/dz = dC/da * da/dz\n",
    "#因为 C = 1/2 * [(y1 - a1)^2 + (y2 - a2)^2 + (y3 - a3)^2] <- 对所有x求和\n",
    "#所以 dC/da = a - y\n",
    "#所以 delta = (a - y) * a(z) * (1 - a(z))\n",
    "\n",
    "#计算卷积层的delta\n",
    "#delta = dC/dz = [dC/dz * dz/dx][输出层] * [da/dz * dz/dx][池化层] * [da/dz][卷积层] <- 对3个过滤器求和\n",
    "#delta = {[dC/dz * dz/dx][输出层] <- 对3个神经元求和} * [da/dz * dz/dx][池化层] * [da/dz][卷积层]\n",
    "\n",
    "#因为 输出层 z = w1*x1 + w2*x2 + w3*x3 + w4*x4 + b\n",
    "#所以 输出层 dz/dx = w\n",
    "\n",
    "#因为 池化层 a = z = max(x1,x2,x3,x4)\n",
    "#所以 池化层 da/dz = 1\n",
    "#所以 池化层 dz/dx = 1:如果x最大 0:如果x不是最大\n",
    "\n",
    "#所以 delta = {[delta(输出层) * w][输出层] <- 对3个神经元求和} * [1:x是最大 0:x不是最大][卷积层] * [a(z) * (1 - a(z))][卷积层]\n",
    "\n",
    "#求梯度\n",
    "#dC/dw = delta * x <- 对所有x求和\n",
    "#dC/db = delta <- 对所有x求和"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((96, 36),\n",
       " (96, 3),\n",
       " array([[0, 0, 0, 1, 0, 0],\n",
       "        [0, 0, 0, 1, 0, 0],\n",
       "        [0, 0, 0, 1, 0, 0],\n",
       "        [0, 0, 0, 1, 0, 0],\n",
       "        [0, 0, 0, 1, 0, 0],\n",
       "        [0, 0, 0, 1, 0, 0]]),\n",
       " array([1, 0, 0]))"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "#加载数据\n",
    "def load_data():\n",
    "    with open('数字123.txt') as fr:\n",
    "        lines = fr.readlines()\n",
    "\n",
    "    x = np.empty((len(lines), 36), dtype=int)\n",
    "    y = np.empty((len(lines), 3), dtype=int)\n",
    "\n",
    "    for i in range(len(lines)):\n",
    "        line = lines[i].strip().split(',')\n",
    "        x[i] = line[:36]\n",
    "        if line[36] == '1':\n",
    "            y[i] = [1, 0, 0]\n",
    "        if line[36] == '2':\n",
    "            y[i] = [0, 1, 0]\n",
    "        if line[36] == '3':\n",
    "            y[i] = [0, 0, 1]\n",
    "\n",
    "    return x, y\n",
    "\n",
    "\n",
    "x, y = load_data()\n",
    "x.shape, y.shape, x[0].reshape(6, 6), y[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#定义常量\n",
    "N, M = x.shape\n",
    "\n",
    "lr = 0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#定义卷积层\n",
    "class Conv:\n",
    "    def __init__(self, w, b):\n",
    "        self.w = w\n",
    "        self.b = b\n",
    "\n",
    "    def run(self, xi):\n",
    "        z = np.zeros((4, 4))\n",
    "        xi = xi.reshape((6, 6))\n",
    "        #扫描xi\n",
    "        for i in range(4):\n",
    "            for j in range(4):\n",
    "                #取xi中的小窗口\n",
    "                window = xi[i:i + 3, j:j + 3]\n",
    "\n",
    "                #线性计算\n",
    "                z[i, j] = np.multiply(window, self.w).sum() + self.b\n",
    "\n",
    "        self.z = z\n",
    "\n",
    "        #激活函数,sigmoid\n",
    "        self.a = 1 / (1 + np.exp(-self.z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#定义池化层\n",
    "class Pool:\n",
    "    def __init__(self):\n",
    "        pass\n",
    "\n",
    "    def run(self, xi):\n",
    "        a = np.zeros((2, 2))\n",
    "\n",
    "        #最大池化\n",
    "        a[0, 0] = xi[0:2, 0:2].max()\n",
    "        a[0, 1] = xi[0:2, 2:4].max()\n",
    "        a[1, 0] = xi[2:4, 0:2].max()\n",
    "        a[1, 1] = xi[2:4, 2:4].max()\n",
    "\n",
    "        self.a = a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#定义输出层\n",
    "class Out:\n",
    "    def __init__(self, w, b):\n",
    "        self.w = w\n",
    "        self.b = b\n",
    "\n",
    "    def run(self, x1, x2, x3):\n",
    "\n",
    "        #线性计算\n",
    "        self.z = np.multiply(x1.reshape(1, -1), self.w[0]).sum()\n",
    "        self.z += np.multiply(x2.reshape(1, -1), self.w[1]).sum()\n",
    "        self.z += np.multiply(x3.reshape(1, -1), self.w[2]).sum()\n",
    "        self.z += self.b\n",
    "\n",
    "        #激活函数,sigmoid\n",
    "        self.a = 1 / (1 + np.exp(-self.z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#初始化卷积层\n",
    "cs = []\n",
    "\n",
    "w = np.array([[-1.277, -0.454, 0.358], [1.138, -2.398, -1.664],\n",
    "              [-0.794, 0.899, 0.675]])\n",
    "cs.append(Conv(w, b=-3.363))\n",
    "\n",
    "w = np.array([[-1.274, 2.338, 2.301], [0.649, -0.339, -2.054],\n",
    "              [-1.022, -1.204, -1.900]])\n",
    "cs.append(Conv(w, b=-3.176))\n",
    "\n",
    "w = np.array([[-1.869, 2.044, -1.290], [-1.710, -2.091, -2.946],\n",
    "              [0.201, -1.323, 0.207]])\n",
    "cs.append(Conv(w, b=-1.739))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#初始化池化层\n",
    "#其实这3个p没有区别,也没有学习的空间,只是为了存储3个a所以new了3个\n",
    "ps = []\n",
    "ps.append(Pool())\n",
    "ps.append(Pool())\n",
    "ps.append(Pool())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#初始化输出层\n",
    "os = []\n",
    "w = np.array([[-0.276, 0.124, -0.961, 0.718], [-3.680, -0.594, 0.280, -0.782],\n",
    "              [-1.475, -2.010, -1.085, -0.188]])\n",
    "os.append(Out(w, b=2.060))\n",
    "\n",
    "w = np.array([[0.010, 0.661, -1.591, 2.189], [1.728, 0.003, -0.250, 1.898],\n",
    "              [0.238, 1.589, 2.246, -0.093]])\n",
    "os.append(Out(w, b=-2.746))\n",
    "\n",
    "w = np.array([[-1.322, -0.218, 3.527, 0.061], [0.613, 0.218, -2.130, -1.678],\n",
    "              [1.236, -0.486, -0.144, -1.235]])\n",
    "os.append(Out(w, b=-1.818))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.7859501531761532, 0.10857818037340734, 0.1369293947699326)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#运算函数\n",
    "def run(xi):\n",
    "    for i in range(3):\n",
    "        cs[i].run(xi)\n",
    "    for i in range(3):\n",
    "        ps[i].run(cs[i].a)\n",
    "    for i in range(3):\n",
    "        os[i].run(ps[0].a, ps[1].a, ps[2].a)\n",
    "\n",
    "\n",
    "run(x[0])\n",
    "os[0].a, os[1].a, os[2].a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.03601014,  0.01050917,  0.01618228],\n",
       "       [-0.08526375,  0.01883231,  0.01726081],\n",
       "       [-0.09935389,  0.02124917,  0.02006905],\n",
       "       [-0.03910314,  0.01076829,  0.01868235],\n",
       "       [-0.03850947,  0.01151791,  0.01831856]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#计算输出层的delta\n",
    "#delta = dC/dz = dC/da * da/dz\n",
    "#因为 C = 1/2 * [(y1 - a1)^2 + (y2 - a2)^2 + (y3 - a3)^2] <- 对所有x求和\n",
    "#所以 dC/da = a - y\n",
    "#所以 delta = (a - y) * a(z) * (1 - a(z))\n",
    "def get_delta_out():\n",
    "    delta_out = np.zeros((N, 3))\n",
    "    for i in range(N):\n",
    "        run(x[i])\n",
    "\n",
    "        for j in range(3):\n",
    "            delta_out[i, j] = (os[j].a - y[i, j]) * os[j].a * (1 - os[j].a)\n",
    "\n",
    "    return delta_out\n",
    "\n",
    "\n",
    "delta_out = get_delta_out()\n",
    "delta_out[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.        ,  0.        ,  0.00695707,  0.        ],\n",
       "       [ 0.07410988,  0.        ,  0.        ,  0.        ],\n",
       "       [-0.00251409, -0.        ,  0.00569069,  0.        ],\n",
       "       [-0.00251409, -0.        ,  0.00569069,  0.        ]])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#计算卷积层的delta\n",
    "#delta = dC/dz = [dC/dz * dz/dx][输出层] * [da/dz * dz/dx][池化层] * [da/dz][卷积层] <- 对3个过滤器求和\n",
    "#delta = {[dC/dz * dz/dx][输出层] <- 对3个神经元求和} * [da/dz * dz/dx][池化层] * [da/dz][卷积层]\n",
    "\n",
    "#因为 输出层 z = w1*x1 + w2*x2 + w3*x3 + w4*x4 + b\n",
    "#所以 输出层 dz/dx = w\n",
    "\n",
    "#因为 池化层 a = z = max(x1,x2,x3,x4)\n",
    "#所以 池化层 da/dz = 1\n",
    "#所以 池化层 dz/dx = 1:如果x最大 0:如果x不是最大\n",
    "\n",
    "\n",
    "#所以 delta = {[delta(输出层) * w][输出层] <- 对3个神经元求和} * [1:x是最大 0:x不是最大][卷积层] * [a(z) * (1 - a(z))][卷积层]\n",
    "def get_delta_conv():\n",
    "    delta_conv = np.zeros((N, 3, 4, 4))\n",
    "\n",
    "    pool_idx = np.array([[0, 0, 1, 1], [0, 0, 1, 1], [2, 2, 3, 3],\n",
    "                         [2, 2, 3, 3]])\n",
    "\n",
    "    #遍历所有数据\n",
    "    for i in range(N):\n",
    "        run(x[i])\n",
    "\n",
    "        #遍历三个卷积器\n",
    "        for j in range(3):\n",
    "\n",
    "            #每个卷机器的delta是一个4x4的矩阵\n",
    "            #扫描这个矩阵\n",
    "            for _1 in range(4):\n",
    "                #print(i, j, _1)\n",
    "                for _2 in range(4):\n",
    "\n",
    "                    #先计算这部分:{[delta(输出层) * w][输出层] <- 对3个神经元求和}\n",
    "                    #输出层的w是一个3x4的矩阵,每个卷机器对应其中的一行\n",
    "                    #pool_idx[_1, _2]是把卷机器4x4的尺寸压缩到2x2的尺寸,因为有池化层\n",
    "                    w_idx = (j, pool_idx[_1, _2])\n",
    "                    sum_delta_out = delta_out[i, 0] * os[0].w[w_idx]\n",
    "                    sum_delta_out += delta_out[i, 1] * os[1].w[w_idx]\n",
    "                    sum_delta_out += delta_out[i, 2] * os[2].w[w_idx]\n",
    "\n",
    "                    #[1:x是最大 0:x不是最大][卷积层]\n",
    "                    d_p = 0\n",
    "                    #判断当前的卷机器输出是否是对应的池化层输出\n",
    "                    if cs[j].a[_1, _2] == ps[j].a[int(_1 / 2), int(_2 / 2)]:\n",
    "                        d_p = 1\n",
    "\n",
    "                    #[a(z) * (1 - a(z))][卷积层]\n",
    "                    #这里就是简单的sigmoid函数求导\n",
    "                    d_a = cs[j].a[_1, _2] * (1 - cs[j].a[_1, _2])\n",
    "\n",
    "                    delta_conv[i, j, _1, _2] = (sum_delta_out * d_p * d_a)\n",
    "\n",
    "    return delta_conv\n",
    "\n",
    "\n",
    "delta_conv = get_delta_conv()\n",
    "delta_conv[1, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[[-0.01717084, -0.22068176, -2.3031552 ],\n",
       "         [-3.46326777,  0.03506276,  0.07279098],\n",
       "         [-1.72280208, -3.67698433, -3.09158066]],\n",
       " \n",
       "        [[-0.14762899, -1.66071179, -0.05313807],\n",
       "         [-1.60017334,  0.4337165 ,  0.32155536],\n",
       "         [ 0.18894448,  0.92748115, -0.34197684]],\n",
       " \n",
       "        [[-0.04400932, -1.21537877, -0.02442836],\n",
       "         [ 0.03122772, -0.22773512,  0.02196469],\n",
       "         [-0.16531662,  0.17641142, -0.64013626]]]),\n",
       " array([-2.98935125, -0.80548239, -1.15646339]))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_gradient_conv():\n",
    "    gradient_conv_w = np.zeros((3, 3, 3))\n",
    "    gradient_conv_b = np.zeros(3)\n",
    "\n",
    "    #遍历所有数据\n",
    "    for i in range(N):\n",
    "\n",
    "        xi = x[i].reshape((6, 6))\n",
    "\n",
    "        #遍历3个卷机器\n",
    "        for j in range(3):\n",
    "\n",
    "            #扫描xi\n",
    "            for _1 in range(3):\n",
    "                for _2 in range(3):\n",
    "                    #取xi中的小窗口\n",
    "                    window = xi[_1:_1 + 4, _2:_2 + 4]\n",
    "\n",
    "                    #求梯度\n",
    "                    #dC/dw = delta * x <- 对所有x求和\n",
    "                    gradient = np.multiply(window, delta_conv[i, j]).sum()\n",
    "                    gradient_conv_w[j, _1, _2] += gradient\n",
    "\n",
    "            #求梯度\n",
    "            #dC/db = delta <- 对所有x求和\n",
    "            gradient_conv_b[j] += delta_conv[i, j].sum()\n",
    "\n",
    "    return gradient_conv_w, gradient_conv_b\n",
    "\n",
    "\n",
    "gradient_conv_w, gradient_conv_b = get_gradient_conv()\n",
    "gradient_conv_w, gradient_conv_b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[[ 0.05746021, -0.04133349,  0.1514548 , -0.0116005 ],\n",
       "         [ 0.23565552, -0.07645565, -0.0514156 ,  0.03793918],\n",
       "         [ 0.17841863, -0.11448917, -0.11509322, -0.12559874]],\n",
       " \n",
       "        [[-0.067052  ,  0.00551467,  0.19821577, -0.3021513 ],\n",
       "         [-1.51468394, -0.164552  ,  0.04691571, -0.9632686 ],\n",
       "         [-1.00859612, -0.32069493, -0.29414387, -0.40720453]],\n",
       " \n",
       "        [[-0.29095766, -0.11754116, -1.15545481,  0.02907475],\n",
       "         [-2.00549104, -0.21869132, -0.18069517, -0.30321599],\n",
       "         [-1.24136192,  0.00422127, -0.04509259, -0.00628449]]]),\n",
       " array([-0.57808623, -1.57259232, -2.5003205 ]))"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def get_gradient_out():\n",
    "    #3个输出神经元 * 3x4w矩阵\n",
    "    gradient_out_w = np.zeros((3, 3, 4))\n",
    "    gradient_out_b = np.zeros(3)\n",
    "\n",
    "    #遍历所有数据\n",
    "    for i in range(N):\n",
    "        run(x[i])\n",
    "\n",
    "        #遍历三个输出神经元\n",
    "        for j in range(3):\n",
    "\n",
    "            #遍历3个池化器\n",
    "            for k in range(3):\n",
    "\n",
    "                #求梯度\n",
    "                #dC/dw = delta * x <- 对所有x求和\n",
    "                gradient = ps[k].a * delta_out[i, j]\n",
    "                gradient = gradient.reshape(-1)\n",
    "\n",
    "                gradient_out_w[j, k] += gradient\n",
    "\n",
    "            #dC/db = delta <- 对所有x求和\n",
    "            gradient_out_b[j] += delta_out[i, j]\n",
    "\n",
    "    return gradient_out_w, gradient_out_b\n",
    "\n",
    "\n",
    "gradient_out_w, gradient_out_b = get_gradient_out()\n",
    "gradient_out_w, gradient_out_b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[-1.27356583, -0.40986365,  0.81863104],\n",
       "        [ 1.83065355, -2.40501255, -1.6785582 ],\n",
       "        [-0.44943958,  1.63439687,  1.29331613]]),\n",
       " -2.765129749407543,\n",
       " array([[-0.28749204,  0.1322667 , -0.99129096,  0.7203201 ],\n",
       "        [-3.7271311 , -0.57870887,  0.29028312, -0.78958784],\n",
       "        [-1.51068373, -1.98710217, -1.06198136, -0.16288025]]),\n",
       " 2.17561724570285)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#更新参数\n",
    "def update():\n",
    "    for i in range(3):\n",
    "        cs[i].w -= gradient_conv_w[i] * lr\n",
    "        cs[i].b -= gradient_conv_b[i] * lr\n",
    "\n",
    "        os[i].w -= gradient_out_w[i] * lr\n",
    "        os[i].b -= gradient_out_b[i] * lr\n",
    "\n",
    "\n",
    "update()\n",
    "\n",
    "cs[0].w, cs[0].b, os[0].w, os[0].b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[-0.64597966, -0.78274848,  0.34882081],\n",
       "        [ 2.39503468, -3.77907531, -2.73777482],\n",
       "        [ 0.72335571,  1.98043273,  1.3402626 ]]),\n",
       " -3.468490587928705,\n",
       " array([[-0.52670233,  0.73697115, -1.78487478,  0.54287261],\n",
       "        [-4.97646234,  0.17952161,  0.55181929, -0.92276147],\n",
       "        [-2.01860614, -1.77628825, -0.8836164 ,  0.03805839]]),\n",
       " 3.1559393950141774)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#批量训练\n",
    "for _ in range(49):\n",
    "    delta_out = get_delta_out()\n",
    "    delta_conv = get_delta_conv()\n",
    "    gradient_conv_w, gradient_conv_b = get_gradient_conv()\n",
    "    gradient_out_w, gradient_out_b = get_gradient_out()\n",
    "    update()\n",
    "    \n",
    "cs[0].w, cs[0].b, os[0].w, os[0].b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#测试\n",
    "correct = 0\n",
    "\n",
    "for i in range(N):\n",
    "    run(x[i])\n",
    "\n",
    "    pred = np.array([os[0].a, os[1].a, os[2].a])\n",
    "    if pred.argmax() == y[i].argmax():\n",
    "        correct += 1\n",
    "\n",
    "correct / N"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
